import requests,random,json,jsonpath,time,pymysql
from weibo.agent import agent
from weibo.randomcook import cookies
from multiprocessing import Pool
# 禁用安全请求警告
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)


headers={
        'user-Agent':random.choice(agent),
        "Host":"m.weibo.cn",
        # "Cookie":"_T_WM=4b5094eddc2a3659cf96b1e5dfbf7d7e; WEIBOCN_FROM=1110005030; SUB=_2A252f_KtDeRhGeNH7lEW9yzJyDWIHXVVg57lrDV6PUJbkdAKLWrMkW1NSrZSGE7s1lWl4B43B1u5k3cAVJ7mf_-3; SUHB=05iK7N7jo7IoBz; SCF=Aq2Nf0NQOuNviOwKLpPwuvDhhFyj4usnTQhCQ7ub4chVUHaQmlalPheLXPsafmnUgf0yaVnIuVCo5-roMWaer8A.; SSOLoginState=1534821117; MLOGIN=1; M_WEIBOCN_PARAMS=luicode%3D20000174%26lfid%3D102803%26uicode%3D10000011%26fid%3D102803",
        'Cookie':random.choice(cookies)
    }
def start_url(page):
    print('正在抓取%s'%page)
    base_url = 'https://m.weibo.cn/api/container/getIndex?containerid=102803&openApp=0&since_id=%s' % page
    html=requests.get(base_url,headers=headers,verify=False)
    html.encoding='utf-8'
    html_list=json.loads(html.text)
    text_list=jsonpath.jsonpath(html_list,'$..mblog')
    for text in text_list:
        username=text['user']['screen_name'] #博主名
        userid = text['user']['id'] #ID
        user_info_url = 'https://m.weibo.cn/profile/info?uid={}'.format(userid)
        dict_list={
            'username':username,
            'userid':userid,
        }
        time.sleep(0.5)
        get_info(user_info_url,dict_list)

def get_info(user_url,dict_list):

    try:
        response=requests.get(user_url,headers=headers,verify=False)
        response.encoding = 'utf-8'
        html_user_info=json.loads(response.text)['data']['user']
        verified_reason = html_user_info.get('verified_reason', '')  # 微博认证
        followers_count = html_user_info['followers_count']  # 粉丝数
        follow_count = html_user_info['follow_count']  # 关注数
        dict_list['verified_reason']=verified_reason
        dict_list['followers_count']=followers_count
        dict_list['follow_count']=follow_count
        time.sleep(0.1)
        save_to_mysql(dict_list)
    except Exception as e:
        pass
# 写入数据库
def save_to_mysql(dict_list):
    coon = pymysql.connect('127.0.0.1', 'root', '123456', 'sina', charset='utf8')
    cursor = coon.cursor()
    try:
        sql = 'insert into sina_weibo(username,userid,verified_reason,followers_count,follow_count)' \
              'values(%s,%s,%s,%s,%s)'
        data = (dict_list['username'], dict_list['userid'], dict_list['verified_reason'], dict_list['followers_count'], dict_list['follow_count'])
        cursor.execute(sql,data)
        print('写入成功%s' % dict_list)
        coon.commit()
    except Exception as e:
        print('原因错误%e'%e)
        pass
    cursor.close()
    coon.close()
if __name__=='__main__':
    # 创建进程
    pool=Pool(4)
    for i in range(1820,10000000):
        pool.apply_async(func=start_url,args=(i,))
    pool.close()  # 关闭进程池，不再接受新的进程
    pool.join()


